
const { BrowserWindow, ipcMain } = require('electron');
const fs = require('fs');
const { dbQuery } = require('../common/db');
const { uuid, readJsonConfigCache, extractErrorInfo } = require("../common/utils");
process.env['ELECTRON_DISABLE_SECURITY_WARNINGS'] = 'true'

/**
 * spider thread
 */
// 运行缓存数据
var cache = {
    tasks: [],
    currTask: null,
    rootUrls: [],
    contentUrlInfos: [],
    spiderWindow: undefined,
    stopTaskIds: [],
    sleepId: -1,
}
var mainWindow = undefined;

function initMainWindow(win) {
    mainWindow = win;
}
/**
 * 采集器主线程执行
 */
async function startSpiderMainThread(mainWin) {
    clearCache();
    mainWindow = mainWin
    showLog("采集线程执行开始!", "debug")
    process.stdout.write("1 ");
    // 无任务时，休息10秒
    if (!hasNextTask()) {
        showLog("hasNextTask=false", "debug")
        sleepId = setTimeout(function () { startSpiderMainThread(mainWin) }, 1000 * 10);
        return;
    }
    var task = nextTask();
    cache.currTask = task;
    // 是否需要停止任务？
    if(await isStop(task)) {
        startSpiderMainThread(mainWin);
        return;
    }
    // 进度
    showLog("开始采集任务：" + task.task_name);
    try {
        await dbQuery("update spider_task set task_status = '3', start_datetime = NOW() where task_id = ?", [task.task_id]);
        mainWindow.webContents.send("task_event", {target: task, type: 'run', setup: 'start', process: 0});
    } catch (e) {
        console.log("查询待采集的任务时异常！", e)
    }
    process.stdout.write("2 ");
    showLog("spiderRootUrls=start", "debug")
    var taskException = false;
    // 采集更多根url，分页url
    // 是否需要停止任务？
    if(await isStop(task)) {
        startSpiderMainThread(mainWin);
        return;
    }
    showLog("采集根地址列表：" + task.root_url)
    if(task.spider_root_url_script){
        var res = await wrapRetry(function () {
            return spiderRootUrls(task);
        }, 3);
        showLog("spiderRootUrls=done", "debug")
        if (res.success) {
            pushRootUrls(res.data);
            mainWindow.webContents.send("task_event", {target: task, type: 'run', setup: 'spiderRootUrls', process: 2});
        } else {
            taskException = true;
            showLog("采集根地址列表失败！" + res.error, "error");
        }
    } else {
        showLog("未配置采集根地址脚本！", "error");
        taskException = true;
    }
    process.stdout.write("3 ");
    // 如果存在根url，并且标志为true时， 继续采集
    if (cache.rootUrls.length == 0) {
        showLog("未采集到新的根地址!");
    }
    var totalRootUrl = cache.rootUrls.length;
    var doneRootUrl = 0;
    if(!task.spider_content_url_script){
        showLog("未配置采集地址脚本！", "error");
        taskException = true;
    } else {
        while (hasNextRootUrl()) {
            // 是否需要停止任务？
            if(await isStop(task)) {
                startSpiderMainThread(mainWin);
                return;
            }
            var rootUrl = nextRootUrl();
            showLog("hasNextRootUrl=" + rootUrl, "debug")
            process.stdout.write("4 ");
            // 采集内容url列表
            showLog("采集URL地址：" + rootUrl)
            res = await wrapRetry(function () {
                return spiderContentUrls(task, rootUrl);
            }, 3);
            doneRootUrl = doneRootUrl + 1;
            if (res.success && res.data) {
                var storeRes = await storeContentUrls(res.data, task);
                if (!storeRes.success) {
                    console.log(storeRes);
                    taskException = true;
                }
                mainWindow.webContents.send("task_event", {target: task, type: 'run', setup: 'spiderUrls', process: 2 + parseInt(doneRootUrl*12/totalRootUrl)});
                await sleep(100);
            } else if (!res.success) {
                taskException = true;
                showLog("采集URL列表失败！" + res.error, "error");
            }
            showLog("spiderContentUrls=done", "debug")
        }
    }
    if (cache.contentUrlInfos.length == 0) {
        showLog("未采集到新的内容地址!");
    }
    var totalContentUrl = cache.contentUrlInfos.length;
    var doneContentUrl = 0;
    // 采集内容数据
    if(!task.spider_content_script){
        showLog("未配置采集内容脚本！", "error");
        taskException = true;
    } else {
        while (hasNextContentUrlInfo()) {
            // 是否需要停止任务？
            if(await isStop(task)) {
                startSpiderMainThread(mainWin);
                return;
            }
            var contentUrlInfo = nextContentUrl();
            process.stdout.write("5 ");
            try {
                var result = await dbQuery(`update spider_record set spider_status = ? where spider_id = ?`, [1, contentUrlInfo.spiderId]);
                showLog("hasNextContentUrlInfo=true", "debug")
                process.stdout.write("6 ");
                if (result.success) {
                    showLog("采集URL：" + contentUrlInfo.url)
                    res = await wrapRetry(function () {
                        return spiderContentData(task, contentUrlInfo);
                    }, 3);
                    doneContentUrl = doneContentUrl + 1;
                    var storeRes = await storeContent(res, contentUrlInfo, task);
                    if (!storeRes.success) {
                        showLog("采集URL内容失败！" + storeRes.error, "error");
                        taskException = true;
                    } else {
                        mainWindow.webContents.send("task_event", {target: task, type: 'run', setup: 'spiderContent', process: 14 + parseInt(doneContentUrl*86/totalContentUrl)});
                        showLog("采集URL内容成功！");
                    }
                    // await sleep(100);
                    showLog("spiderContentData=done", "debug")
                } else {
                    taskException = true;
                    showLog(`采集(${JSON.stringify(contentUrlInfo)})内容异常,插入采集任务到数据库异常！${JSON.stringify(result)}`, 'error');
                }
            } catch (e) {
                showLog("采集内容异常！" + extractErrorInfo(e))
                console.log("采集内容异常：", e)
                taskException = true;
            }
        }
    }
    process.stdout.write("7 ");
    try {
        await dbQuery("update spider_task set task_status = ?, end_datetime = NOW() where task_id = ?", [taskException ? '4' : '5', task.task_id]);
        mainWindow.webContents.send("task_event", {target: task, type: 'run', setup: 'spiderDone', process: 100});
    } catch (e) {
        console.log("更新任务状态为结束异常！", e)
        mainWindow.webContents.send("task_event", {target: task, type: 'run', setup: 'spiderDone', process: 100});
    }
    process.stdout.write("8 ");
    startSpiderMainThread(mainWin);
}

async function isStop(task) {
    if(inArray(task.task_id, cache.stopTaskIds)) {
        var result = await dbQuery("update spider_task set task_status = '7', start_datetime = NOW(), end_datetime = NOW() where task_id = ?", [task.task_id]);
        if(result.success) {
            removeStopTask(task.task_id)
            showLog(`任务已停止:${task.task_name}`)
            mainWindow.webContents.send("task_event", {target: task, type: 'stop', setup: 'stop', process: 0});
            return true;
        } else {
            showLog(`任务已停止失败，${result.error}！${task.task_name}`)
            return true
        }
    }
    return false;
}

function removeStopTask(taskId){
    var array = cache.stopTaskIds;
    var newArray = []
    for (let index = 0; index < array.length; index++) {
        if(array[index] != taskId) {
            newArray[newArray.length] = array[index];
        }
    }
    cache.stopTaskIds = newArray;
}

/**
 * 存储采集内容
 * 
 * @param {*} result 
 * @param {*} contentUrlInfo 
 */
async function storeContent(result, contentUrlInfo, task) {
    if (result.success) {
        showLog("结果：" + JSON.stringify(result.data));
        try {
            // save content to file
            if (!saveContentToFile(result, contentUrlInfo, task)) {
                return { success: false, error: "保存采集内容到文件失败！" };
            }
            var res = await dbQuery("update spider_record set spider_title = ?, spider_content = ?, spider_status = '3', spider_datetime=NOW() where spider_id = ? ", [result.data.title, JSON.stringify(result.data), contentUrlInfo.spiderId]);
            if (res.success && res.data.affectedRows > 0) {
                return { success: true, msg: "更新URL内容结果成功！" };
            } else {
                return { success: false, error: "更新URL内容结果失败！" }
            }
        } catch (e) {
            console.log(e);
            var errorInfo = extractErrorInfo(e);
            showLog("更新采集内容异常!" + errorInfo, 'error');
            try {
                await dbQuery("update spider_record set spider_title = ?, spider_content = ?,spider_status = '2', spider_datetime=NOW() where spider_id = ? ", ["采集异常", "更新采集内容异常!" + errorInfo, contentUrlInfo.spiderId]);
            } catch (e) {
                console.log(e);
            }
            return { success: false, error: "更新采集内容异常!" + errorInfo }
        }
    } else {
        try {
            await dbQuery("update spider_record set spider_title = ?, spider_content = ?,spider_status = '2' where spider_id = ? ", ["存储异常", result.error, contentUrlInfo.spiderId]);
        } catch (e) {
            console.log(e);
        }
        return { success: false, error: "存储采集内容异常!" }
    }
}

/**
 * 保存采集内容到文件中
 * 
 * @param {*} result 
 * @param {*} contentUrlInfo 
 * @param {*} task 
 */
function saveContentToFile(result, contentUrlInfo, task) {
    var configJson = readJsonConfigCache("common.config.json");
    var storePath = configJson.storePath;
    if (!storePath) {
        storePath = process.cwd() + "/data/spider/";
    }
    var path = storePath + task.task_id + "/" + contentUrlInfo.spiderId + "/";
    if (!fs.existsSync(path)) {
        fs.mkdirSync(path, { recursive: true });
    }
    try {
        fs.writeFileSync(path + "content.txt", JSON.stringify(result.data));
        return true;
    } catch (e) {
        console.log("写入采集内容到文件异常！", e)
    }
    return false;
}

// 存储采集到的内容url
async function storeContentUrls(urls, task) {
    var error = false;
    // 采集到urls，判断，缓存
    for (var idx = 0; idx < urls.length; idx++) {
        var url = urls[idx];
        try {
            var dbRes = await dbQuery("select * from spider_record where spider_url = ? limit 1", [url]);
            // 未采集过的内容url
            if (dbRes.success && (!dbRes.data || dbRes.data.length <= 0)) {
                // 不重复时
                if (!inArray({ url: url }, cache.contentUrlInfos, 'url')) {
                    var spiderId = uuid();
                    try {
                        await dbQuery("insert into spider_record(spider_id, task_id, spider_url, spider_status, spider_datetime) values (?,?,?,?,NOW())",
                            [spiderId, task.task_id, url, '0'])
                        process.stdout.write("42 ");
                    } catch (e) {
                        showLog("插入采集记录异常：" + extractErrorInfo(e), 'error');
                        error = true;
                    }
                    process.stdout.write("43 ");
                    pushContentUrlInfos({ spiderId: spiderId, url: url });
                }
            } else if (dbRes.success && dbRes.data && dbRes.data.length > 0) {
                var spiderRecord = dbRes.data[0];
                // 如果存在的记录状态不是成功
                if (spiderRecord.spider_status != 3) {
                    // 不重复时
                    if (!inArray({ url: spiderRecord.spider_url }, cache.contentUrlInfos, 'url')) {
                        pushContentUrlInfos({ spiderId: spiderRecord.spider_id, url: spiderRecord.spider_url });
                    }
                    showLog("已存在URL: " + url + ", 内容未采集，准备采集！");
                } else {
                    showLog("已存在URL: " + url + ", 内容已采集，跳过！");
                }
            }
        } catch (e) {
            showLog("存储URL信息异常：" + extractErrorInfo(e), 'error');
            console.log("存储URL信息异常", e);
            error = true;
        }
    }
    return {success: !error};
}

async function wrapRetry(fn, retryCount) {
    var res = await fn();
    var count = retryCount;
    while (!res.success && count > 0) {
        // sleep some time
        await sleep(3000);
        showLog(res.error + " 正在重试: " + count, 'error');
        count = count - 1;
        res = await fn();
    }
    return res;
}

function sleep(time) {
    return new Promise(resolve => {
        setTimeout(function () {
            resolve(true)
        }, time)
    });
}

function getSpiderWindow() {
    if (!cache.spiderWindow || cache.spiderWindow.isDestroyed()) {
        var win = new BrowserWindow({
            width: 1200,
            height: 900,
            webPreferences: {
                nodeIntegration: false,
                nodeIntegrationInWorker: false,
                nodeIntegrationInSubFrames: false,
                contextIsolation: false,
                show: false,
                closable: false,

            }
        })
        win.hide();
        cache.spiderWindow = win;
    }
    return cache.spiderWindow;
}
/**
 * 采集任务的根url
 * @param  task 
 */
function spiderRootUrls(task) {
    task.rootUrlLoading = true;
    var getRootUrls = function (resolve, result) {
        process.stdout.write("21 ");
        if (result.success) {
            resolve({ success: true, data: result.data });
            return;
        }
        resolve({ success: false, error: "采集根地址脚本执行异常" + result.error });
    }
    var timeout = task.spider_timeout?task.spider_timeout:3000;
    var p = new Promise(resolve => {
        getSpiderWindow().webContents.removeAllListeners("did-stop-loading").once("did-stop-loading", function () {
            getSpiderWindow().webContents.executeJavaScript(wrapScript(task.spider_root_url_script), true)
                .then((v) => {
                    console.log("execjs", v)
                    getRootUrls(resolve, v)
                    task.rootUrlLoading = false;
                }).catch(e => {
                    resolve({ success: false, error: "采集根地址脚本语法不正确" + extractErrorInfo(e) });
                    task.rootUrlLoading = false;
                });
        })
    });
    getSpiderWindow().loadURL(task.root_url); 
    setTimeout(function () {
        var result = { success: false, error: "页面加载超时("+timeout+")！" + task.root_url };
        if(getSpiderWindow().webContents.isLoading() && task.rootUrlLoading) {
            showLog("页面加载超时("+timeout+")，停止加载并尝试采集！  " + task.root_url, 'error')
            getSpiderWindow().webContents.stop();
        }
    }, timeout);
    var pt = new Promise(resolve => {
        setTimeout(function () {
            var result = ({ success: false, error: "页面执行超时("+timeout*3+")！" + task.root_url });
            if(getSpiderWindow().webContents.isLoading() && task.rootUrlLoading) {
                console.log("exec timeout ... ")
                showLog("页面执行超时，采集失败" + task.root_url, 'error')
            }
            resolve(result) ;
        }, timeout*3);
    });
    return Promise.race([p, pt]);
    // return p;
}


function wrapScript(script, triggerScene) {
    var wrap = `
        try{
            var result = (function (){ 
                var _trigger_scene_ = '${triggerScene ? triggerScene : ''}';
                ${script}
            })();
            var a = {success: true, data: result};a;
        }catch(e) {
            var a = {success: false, error: e};a;
        }
    `;
    return wrap;
}



/**
 * 采集任务的内容url
 * @param  task 
 */
function spiderContentUrls(task, rootUrl) {
    task["_" + rootUrl] = true;
    var getContentUrls = async function (resolve, result) {
        process.stdout.write("41 ");
        try {
            if (!result.success) {
                    ({ success: false, error: "采集地址脚本执行异常:" + result.error });
                return;
            }
            var newResult = new Array();
            var flag = true;
            process.stdout.write("44 ");
            resolve({ success: true, data: result.data });
        } catch (e) {
            showLog("解析内容URL数据异常！" + e, "error");
            resolve({ success: false, error: "解析内容URL数据异常！" + e });
        }
    }
    var timeout = task.spider_timeout?task.spider_timeout:3000;
    var p = new Promise((resolve) => {
        getSpiderWindow().webContents.removeAllListeners("did-stop-loading").once("did-stop-loading", function () {
            getSpiderWindow().webContents.executeJavaScript(wrapScript(task.spider_content_url_script), true)
                .then((v) => {
                    getContentUrls(resolve, v)
                    task["_" + rootUrl] = false;
                }).catch(e => {
                    process.stdout.write("45 ");
                    showLog("采集地址脚本语法不正确: " + task.spider_content_url_script, "error")
                    resolve({ success: false, error: "采集地址脚本执行异常！" + e })
                    task["_" + rootUrl] = false;
                });
        })
    })
    getSpiderWindow().loadURL(rootUrl);
    setTimeout(function () {
        var result = ({ success: false, error: "页面加载超时("+timeout+")！" + rootUrl });
        if(getSpiderWindow().webContents.isLoading() && task["_" + rootUrl]) {
            showLog("页面加载超时("+timeout+")，停止加载并尝试采集！" + rootUrl, 'error')
            getSpiderWindow().webContents.stop();
        }
    }, timeout);
    var pt = new Promise(resolve => {
        setTimeout(function () {
            var result = ({ success: false, error: "页面执行超时("+timeout*3+")！" + rootUrl });
            if(getSpiderWindow().webContents.isLoading() && task["_" + rootUrl]) {
                console.log("exec timeout ... ")
                showLog("页面执行超时，采集失败" + rootUrl, 'error')
            }
            resolve(result) ;
        }, timeout*3);
    })
    return Promise.race([p, pt]);
    // return p;
}
/**
 * 判断目标是否在数组中
 * 
 * @param {*} target 
 * @param {*} arrays 
 * @param {*} key [option] 
 * @returns 
 */
function inArray(target, arrays, key) {
    if (target && arrays) {
        for (var i = 0; i < arrays.length; i++) {
            if (key) {
                if (target[key] == arrays[i][key]) {
                    return true;
                }
            } else {
                if (target == arrays[i]) {
                    return true;
                }
            }
        }
    }
    return false;
}

/**
 * 采集url的内容数据
 * @param  task 
 */
function spiderContentData(task, contentUrlInfo) {
    contentUrlInfo.isLoading = true;
    var getData = async function (resolve, result) {
        process.stdout.write("61 ");
        try {
            if (!result.success) {
                showLog("采集内容脚本执行异常:" + result.error, "error");
                resolve({ success: false, error: result.error });
                return;
            }
            process.stdout.write("62 ");
            resolve(result);
        } catch (e) {
            console.log("解析采集结果异常！", e);
            showLog("解析采集结果异常！" + e, "error");
            resolve({ success: false, error: "解析采集结果异常！" + extractErrorInfo(e) });
        }
    }
    var timeout = task.spider_timeout?task.spider_timeout:3000;
    var p = new Promise(resolve => {
        getSpiderWindow().webContents.removeAllListeners("did-stop-loading").once("did-stop-loading", function () {
            console.log('did-stop-loading')
            try{
                getSpiderWindow().webContents.executeJavaScript(wrapScript(task.spider_content_script), true)
                .then((v) => {
                    getData(resolve, v);
                    contentUrlInfo.isLoading = false;
                })
                .catch(e => {
                    process.stdout.write("60 ");
                    showLog("采集内容脚本不正确:" + task.spider_content_script, "error");
                    resolve({ success: false, error: "采集内容脚本执行异常！" + extractErrorInfo(e) });
                    contentUrlInfo.isLoading = false;
                });
            }catch(e) {
                showLog("调用采集内容脚本异常:" + extractErrorInfo(e), "error");
                console.log("调用采集内容脚本异常", e)
                resolve({success: false, error: "调用采集内容脚本异常:" + extractErrorInfo(e)})
            }
        })
    })
    getSpiderWindow().loadURL(contentUrlInfo.url)
    setTimeout(function () {
        console.log("timeout ... ")
        var result = ({ success: false, error: "页面加载超时("+timeout+")！" + contentUrlInfo.url });
        if(getSpiderWindow().webContents.isLoading() && contentUrlInfo.isLoading) {
            showLog("页面加载超时("+timeout+")，停止加载并尝试采集！" + contentUrlInfo.url, 'error')
            getSpiderWindow().webContents.stop();
        }
    }, timeout);
    var pt = new Promise(resolve => {
        setTimeout(function () {
            var result = ({ success: false, error: "页面执行超时("+timeout*3+")！" + contentUrlInfo.url });
            if(getSpiderWindow().webContents.isLoading() && contentUrlInfo.isLoading) {
                console.log("exec timeout ... ")
                showLog("页面执行超时，采集失败" + contentUrlInfo.url, 'error')
            }
            resolve(result) ;
        }, timeout*3);
    });
    return Promise.race([p, pt]);
    // return p;
}

function showLog(msg, type) {
    mainWindow.webContents.send("log", { msg: msg, type: type });
}

async function stopTask(task) {
    var result = null;
    var isStoped = false;
    // 如果执行的任务状态在执行队列中
    if(inArray(task.task_status, [2,3])) {
        var pending = JSON.parse(JSON.stringify(cache.tasks));
        if(cache.currTask) {
            pending[pending.length] = cache.currTask;
        }
        if(!inArray(task, pending, "task_id")) {
            // 如果任务不在队列或运行
            result = await dbQuery("update spider_task set task_status = '7', start_datetime = NOW() where task_id = ?", [task.task_id]);
            isStoped = true;
        } else {
            // 如果任务在队列或运行
            result = await dbQuery("update spider_task set task_status = '6', start_datetime = NOW() where task_id = ? and task_status in (2,3)", [task.task_id]);
        }
    } else {
        // 如果任务已经停止，直接停止
        result = await dbQuery("update spider_task set task_status = '7', start_datetime = NOW() where task_id = ? and task_status in (0, 4, 5, 6, 7)", [task.task_id]);
        if(result.success && result.data.affectedRows == 1) {
            console.log('stoped task!', result)
            isStoped = true;
        }
        // 如果任务正在运行
        result = await dbQuery("update spider_task set task_status = '6', start_datetime = NOW() where task_id = ? and task_status in (2,3)", [task.task_id]);
    }
    if(result.success) {
        showLog(`已发送任务停止信号 => 《${task.task_name}》`)
        if(!isStoped) {
            cache.stopTaskIds[cache.stopTaskIds.length] = task.task_id;
        }
        mainWindow.webContents.send("task_event", {target: task, type: 'stop', setup: 'stoping', process: 0});
    } else {
        showLop(`发送任务停止信号失败！ => 《task.task_name》。 ${result.error}`)
        cache.stopTaskIds[cache.stopTaskIds.length] = task.task_id;
    }
    return result;
}

function pushRootUrls(rootUrls) {
    if (!rootUrls) {
        return;
    }
    if (Array.isArray(rootUrls)) {
        for (var i = 0; i < rootUrls.length; i++) {
            if (!inArray(rootUrls[i], cache.rootUrls)) {
                showLog("采集到根地址：" + rootUrls[i]);
                cache.rootUrls[cache.rootUrls.length] = rootUrls[i];
            }
        }
    } else {
        if (!inArray(rootUrls, cache.rootUrls)) {
            showLog("采集到根地址：" + rootUrls);
            cache.rootUrls[cache.rootUrls.length] = rootUrls;
        }
    }
}

function uniqueArray(arr) {
    return Array.from(new Set(arr))
}

function pushContentUrlInfos(contentUrlInfos) {
    if (!contentUrlInfos) {
        return;
    }
    if (Array.isArray(contentUrlInfos)) {
        for (var i = 0; i < contentUrlInfos.length; i++) {
            if (contentUrlInfos[i]) {
                if (!inArray(contentUrlInfos[i], cache.contentUrlInfos, 'url')) {
                    showLog("采集到URL:  " + contentUrlInfos[i].url);
                    cache.contentUrlInfos[cache.contentUrlInfos.length] = contentUrlInfos[i];
                }
            }
        }
    } else {
        if (!inArray(contentUrlInfos, cache.contentUrlInfos, 'url')) {
            showLog("采集到URL:  " + contentUrlInfos.url);
            cache.contentUrlInfos[cache.contentUrlInfos.length] = contentUrlInfos;
        }
    }
    cache.contentUrlInfos = uniqueArray(cache.contentUrlInfos);
}

function hasNextRootUrl() {
    return cache.rootUrls.length > 0;
}

function nextRootUrl() {
    const rootUrl = cache.rootUrls[0]
    cache.rootUrls = cache.rootUrls.slice(1);
    return rootUrl;
}

function hasNextContentUrlInfo() {
    return cache.contentUrlInfos.length > 0;
}

function nextContentUrl() {
    const contentUrlInfo = cache.contentUrlInfos[0]
    cache.contentUrlInfos = cache.contentUrlInfos.slice(1);
    return contentUrlInfo;
}

function hasNextTask() {
    return cache.tasks.length > 0;
}
function nextTask() {
    const task = cache.tasks[0]
    cache.tasks = cache.tasks.slice(1);
    return task;
}

async function pushTasks(tasks) {
    if (!tasks) {
        return;
    }
    if (Array.isArray(tasks)) {
        for (var i = 0; i < tasks.length; i++) {
            cache.tasks[cache.tasks.length] = tasks[i];
            await dbQuery("update spider_task set task_status = ? where task_id = ?", [2, tasks[i].task_id])
        }
    } else {
        cache.tasks[cache.tasks.length] = tasks;
        await dbQuery("update spider_task set task_status = ? where task_id = ?", [2, tasks.task_id])
    }
    console.log("startTasks", tasks)
}

async function startTasks(tasks) {
    await pushTasks(tasks);
}

function clearCache() {
    if (cache.sleepId > 0) {
        clearTimeout(cache.sleepId);
        cache.sleepId = -1;
    }
    cache.rootUrls = [];
    cache.contentUrlInfos = [];
    cache.currTask = null;
}


module.exports = {
    startSpiderMainThread: startSpiderMainThread,
    startTasksAsync: startTasks,
    stopTaskAsync: stopTask,
    initMainWindow: initMainWindow,
    getSpiderWindow: getSpiderWindow,

}
